{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Format DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(600, 51)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x_0</th>\n",
       "      <th>x_1</th>\n",
       "      <th>x_2</th>\n",
       "      <th>x_3</th>\n",
       "      <th>x_4</th>\n",
       "      <th>x_5</th>\n",
       "      <th>x_6</th>\n",
       "      <th>x_7</th>\n",
       "      <th>x_8</th>\n",
       "      <th>x_9</th>\n",
       "      <th>...</th>\n",
       "      <th>x_41</th>\n",
       "      <th>x_42</th>\n",
       "      <th>x_43</th>\n",
       "      <th>x_44</th>\n",
       "      <th>x_45</th>\n",
       "      <th>x_46</th>\n",
       "      <th>x_47</th>\n",
       "      <th>x_48</th>\n",
       "      <th>x_49</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.705718</td>\n",
       "      <td>-0.131674</td>\n",
       "      <td>-1.323795</td>\n",
       "      <td>1.136606</td>\n",
       "      <td>-0.928763</td>\n",
       "      <td>0.284816</td>\n",
       "      <td>-1.187778</td>\n",
       "      <td>-3.055318</td>\n",
       "      <td>-0.212013</td>\n",
       "      <td>1.671612</td>\n",
       "      <td>...</td>\n",
       "      <td>0.410953</td>\n",
       "      <td>0.755633</td>\n",
       "      <td>1.088105</td>\n",
       "      <td>0.111611</td>\n",
       "      <td>-0.614697</td>\n",
       "      <td>-0.207736</td>\n",
       "      <td>0.179674</td>\n",
       "      <td>-0.231539</td>\n",
       "      <td>0.767044</td>\n",
       "      <td>-124.517151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.713429</td>\n",
       "      <td>-1.736042</td>\n",
       "      <td>0.767225</td>\n",
       "      <td>0.745636</td>\n",
       "      <td>-1.876690</td>\n",
       "      <td>-0.712041</td>\n",
       "      <td>0.229587</td>\n",
       "      <td>-0.058826</td>\n",
       "      <td>-2.344972</td>\n",
       "      <td>0.654297</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.274046</td>\n",
       "      <td>-0.340395</td>\n",
       "      <td>0.085439</td>\n",
       "      <td>-0.241921</td>\n",
       "      <td>1.369061</td>\n",
       "      <td>1.652834</td>\n",
       "      <td>0.265328</td>\n",
       "      <td>-0.184885</td>\n",
       "      <td>-0.238244</td>\n",
       "      <td>-189.821389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.322404</td>\n",
       "      <td>-1.008531</td>\n",
       "      <td>0.080041</td>\n",
       "      <td>0.243827</td>\n",
       "      <td>0.441083</td>\n",
       "      <td>0.301931</td>\n",
       "      <td>1.414744</td>\n",
       "      <td>-0.287359</td>\n",
       "      <td>0.848805</td>\n",
       "      <td>-0.608154</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.306996</td>\n",
       "      <td>1.363546</td>\n",
       "      <td>-1.992186</td>\n",
       "      <td>-0.690337</td>\n",
       "      <td>-0.246858</td>\n",
       "      <td>1.112149</td>\n",
       "      <td>0.256325</td>\n",
       "      <td>-0.505621</td>\n",
       "      <td>-0.065529</td>\n",
       "      <td>-117.707136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.147538</td>\n",
       "      <td>-0.756780</td>\n",
       "      <td>1.085672</td>\n",
       "      <td>-1.070508</td>\n",
       "      <td>0.289830</td>\n",
       "      <td>-0.148429</td>\n",
       "      <td>-2.815124</td>\n",
       "      <td>-2.617113</td>\n",
       "      <td>1.057043</td>\n",
       "      <td>-0.225982</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.245819</td>\n",
       "      <td>0.809846</td>\n",
       "      <td>1.387846</td>\n",
       "      <td>0.492437</td>\n",
       "      <td>1.475454</td>\n",
       "      <td>-0.334773</td>\n",
       "      <td>-0.230849</td>\n",
       "      <td>0.824702</td>\n",
       "      <td>-0.202968</td>\n",
       "      <td>-205.799334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.343111</td>\n",
       "      <td>0.093387</td>\n",
       "      <td>-1.988132</td>\n",
       "      <td>-1.796316</td>\n",
       "      <td>-2.628521</td>\n",
       "      <td>-0.493052</td>\n",
       "      <td>-0.788237</td>\n",
       "      <td>-0.470098</td>\n",
       "      <td>0.262020</td>\n",
       "      <td>-1.088517</td>\n",
       "      <td>...</td>\n",
       "      <td>0.561577</td>\n",
       "      <td>-0.518497</td>\n",
       "      <td>0.426443</td>\n",
       "      <td>1.003505</td>\n",
       "      <td>0.201326</td>\n",
       "      <td>-1.052442</td>\n",
       "      <td>0.353306</td>\n",
       "      <td>-1.566265</td>\n",
       "      <td>0.532124</td>\n",
       "      <td>99.591967</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        x_0       x_1       x_2       x_3       x_4       x_5       x_6  \\\n",
       "0  1.705718 -0.131674 -1.323795  1.136606 -0.928763  0.284816 -1.187778   \n",
       "1  0.713429 -1.736042  0.767225  0.745636 -1.876690 -0.712041  0.229587   \n",
       "2  1.322404 -1.008531  0.080041  0.243827  0.441083  0.301931  1.414744   \n",
       "3 -0.147538 -0.756780  1.085672 -1.070508  0.289830 -0.148429 -2.815124   \n",
       "4  0.343111  0.093387 -1.988132 -1.796316 -2.628521 -0.493052 -0.788237   \n",
       "\n",
       "        x_7       x_8       x_9     ...          x_41      x_42      x_43  \\\n",
       "0 -3.055318 -0.212013  1.671612     ...      0.410953  0.755633  1.088105   \n",
       "1 -0.058826 -2.344972  0.654297     ...     -0.274046 -0.340395  0.085439   \n",
       "2 -0.287359  0.848805 -0.608154     ...     -0.306996  1.363546 -1.992186   \n",
       "3 -2.617113  1.057043 -0.225982     ...     -0.245819  0.809846  1.387846   \n",
       "4 -0.470098  0.262020 -1.088517     ...      0.561577 -0.518497  0.426443   \n",
       "\n",
       "       x_44      x_45      x_46      x_47      x_48      x_49      target  \n",
       "0  0.111611 -0.614697 -0.207736  0.179674 -0.231539  0.767044 -124.517151  \n",
       "1 -0.241921  1.369061  1.652834  0.265328 -0.184885 -0.238244 -189.821389  \n",
       "2 -0.690337 -0.246858  1.112149  0.256325 -0.505621 -0.065529 -117.707136  \n",
       "3  0.492437  1.475454 -0.334773 -0.230849  0.824702 -0.202968 -205.799334  \n",
       "4  1.003505  0.201326 -1.052442  0.353306 -1.566265  0.532124   99.591967  \n",
       "\n",
       "[5 rows x 51 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.datasets import make_regression\n",
    "\n",
    "data = make_regression(n_samples=600, n_features=50, noise=0.1, random_state=42)\n",
    "train_df = pd.DataFrame(data[0], columns=[\"x_{}\".format(_) for _ in range(data[0].shape[1])])\n",
    "train_df[\"target\"] = data[1]\n",
    "\n",
    "print(train_df.shape)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set Up Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Experiment Key:   'Sxkz36nLbTyi4QJM7w6wCWkbIucXm0RbzMsirLPuYmw='\n"
     ]
    }
   ],
   "source": [
    "from hyperparameter_hunter import Environment, CVExperiment\n",
    "from sklearn.metrics import explained_variance_score\n",
    "\n",
    "env = Environment(\n",
    "    train_dataset=train_df,\n",
    "    results_path=\"HyperparameterHunterAssets\",\n",
    "    metrics=dict(evs=explained_variance_score),\n",
    "    cv_type=\"KFold\",\n",
    "    cv_params=dict(n_splits=3, shuffle=True, random_state=1337),\n",
    "    runs=2,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that HyperparameterHunter has an active `Environment`, we can do two things:\n",
    "\n",
    "# 1. Perform Experiments\n",
    "\n",
    "*Note: If this is your first HyperparameterHunter example, the CatBoost classification example may be a better starting point.*\n",
    "\n",
    "In this Experiment, we're also going to use `model_extra_params` to provide arguments to `CatBoostRegressor`'s `fit` method, just like we would if we weren't using HyperparameterHunter.\n",
    "\n",
    "We'll be using the `verbose` argument to print evaluations of our `CatBoostRegressor` every 50 iterations, and we'll also be using the dataset sentinels offered by `Environment`. You can read more about the exciting thing you can do with the `Environment` sentinels in the documentation and in the example dedicated to them. For now, though, we'll be using them to provide each fold's `env.validation_input`, and `env.validation_target` to `CatBoostRegressor.fit` via its `eval_set` argument.\n",
    "\n",
    "You could also easily add `CatBoostRegressor.fit`'s `early_stopping_rounds` argument to `model_extra_params[\"fit\"]` to use early stopping, but doing so here with only `iterations=100` doesn't make much sense."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<22:13:22> Validated Environment:  'Sxkz36nLbTyi4QJM7w6wCWkbIucXm0RbzMsirLPuYmw='\n",
      "<22:13:22> Initialized Experiment: 'df2982f6-ab54-4c74-80b6-ad1c1e49fe45'\n",
      "<22:13:22> Hyperparameter Key:     '913_iDDPY_PMp5ulOyK251mgLCZ2qau7kjOvDdy1rz8='\n",
      "<22:13:22> \n",
      "0:\tlearn: 181.0383680\ttest: 179.9132919\tbest: 179.9132919 (0)\ttotal: 63ms\tremaining: 6.24s\n",
      "50:\tlearn: 97.8034574\ttest: 110.2078251\tbest: 110.2078251 (50)\ttotal: 427ms\tremaining: 410ms\n",
      "99:\tlearn: 65.3572502\ttest: 86.2235253\tbest: 86.2235253 (99)\ttotal: 783ms\tremaining: 0us\n",
      "\n",
      "bestTest = 86.22352531\n",
      "bestIteration = 99\n",
      "\n",
      "<22:13:22> F0/R0  |  OOF(evs=0.77530)  |  Time Elapsed: 0.81737 s\n",
      "0:\tlearn: 180.3273426\ttest: 178.7632887\tbest: 178.7632887 (0)\ttotal: 7.5ms\tremaining: 742ms\n",
      "50:\tlearn: 95.3065291\ttest: 108.1124182\tbest: 108.1124182 (50)\ttotal: 373ms\tremaining: 358ms\n",
      "99:\tlearn: 63.7322170\ttest: 85.4546004\tbest: 85.4546004 (99)\ttotal: 724ms\tremaining: 0us\n",
      "\n",
      "bestTest = 85.45460041\n",
      "bestIteration = 99\n",
      "\n",
      "<22:13:23> F0/R1  |  OOF(evs=0.77929)  |  Time Elapsed: 0.73847 s\n",
      "<22:13:23> F0.0 AVG:   OOF(evs=0.77998)  |  Time Elapsed: 1.56024 s\n",
      "0:\tlearn: 183.5077544\ttest: 173.1917854\tbest: 173.1917854 (0)\ttotal: 7.24ms\tremaining: 717ms\n",
      "50:\tlearn: 100.5844263\ttest: 108.1710145\tbest: 108.1710145 (50)\ttotal: 375ms\tremaining: 360ms\n",
      "99:\tlearn: 67.5414394\ttest: 85.5053407\tbest: 85.5053407 (99)\ttotal: 734ms\tremaining: 0us\n",
      "\n",
      "bestTest = 85.5053407\n",
      "bestIteration = 99\n",
      "\n",
      "<22:13:24> F1/R0  |  OOF(evs=0.76335)  |  Time Elapsed: 0.7484 s\n",
      "0:\tlearn: 183.4412848\ttest: 172.7748236\tbest: 172.7748236 (0)\ttotal: 7.37ms\tremaining: 730ms\n",
      "50:\tlearn: 98.9207169\ttest: 107.3348632\tbest: 107.3348632 (50)\ttotal: 372ms\tremaining: 357ms\n",
      "99:\tlearn: 67.1194369\ttest: 84.1135630\tbest: 84.1135630 (99)\ttotal: 728ms\tremaining: 0us\n",
      "\n",
      "bestTest = 84.11356298\n",
      "bestIteration = 99\n",
      "\n",
      "<22:13:25> F1/R1  |  OOF(evs=0.77169)  |  Time Elapsed: 0.74292 s\n",
      "<22:13:25> F0.1 AVG:   OOF(evs=0.77061)  |  Time Elapsed: 1.49607 s\n",
      "0:\tlearn: 176.0961712\ttest: 188.8149185\tbest: 188.8149185 (0)\ttotal: 6.9ms\tremaining: 683ms\n",
      "50:\tlearn: 94.2771704\ttest: 120.4426175\tbest: 120.4426175 (50)\ttotal: 375ms\tremaining: 360ms\n",
      "99:\tlearn: 67.9729670\ttest: 99.9417843\tbest: 99.9417843 (99)\ttotal: 730ms\tremaining: 0us\n",
      "\n",
      "bestTest = 99.94178427\n",
      "bestIteration = 99\n",
      "\n",
      "<22:13:25> F2/R0  |  OOF(evs=0.72638)  |  Time Elapsed: 0.74454 s\n",
      "0:\tlearn: 177.0392612\ttest: 189.8521136\tbest: 189.8521136 (0)\ttotal: 6.98ms\tremaining: 691ms\n",
      "50:\tlearn: 95.5295037\ttest: 121.3560567\tbest: 121.3560567 (50)\ttotal: 377ms\tremaining: 362ms\n",
      "99:\tlearn: 66.3592641\ttest: 99.0826473\tbest: 99.0826473 (99)\ttotal: 736ms\tremaining: 0us\n",
      "\n",
      "bestTest = 99.08264731\n",
      "bestIteration = 99\n",
      "\n",
      "<22:13:26> F2/R1  |  OOF(evs=0.73186)  |  Time Elapsed: 0.75069 s\n",
      "<22:13:26> F0.2 AVG:   OOF(evs=0.73123)  |  Time Elapsed: 1.49972 s\n",
      "<22:13:26> \n",
      "<22:13:26> FINAL:    OOF(evs=0.75971)  |  Time Elapsed: 4.56196 s\n",
      "<22:13:26> \n",
      "<22:13:26> Saving results for Experiment: 'df2982f6-ab54-4c74-80b6-ad1c1e49fe45'\n"
     ]
    }
   ],
   "source": [
    "from catboost import CatBoostRegressor\n",
    "\n",
    "experiment = CVExperiment(\n",
    "    model_initializer=CatBoostRegressor,\n",
    "    model_init_params=dict(\n",
    "        iterations=100,\n",
    "        learning_rate=0.05,\n",
    "        depth=5,\n",
    "        bootstrap_type=\"Bayesian\",\n",
    "        save_snapshot=False,\n",
    "        allow_writing_files=False,\n",
    "    ),\n",
    "    model_extra_params=dict(\n",
    "        fit=dict(\n",
    "            verbose=50,\n",
    "            eval_set=[(env.validation_input, env.validation_target)],\n",
    "        ),\n",
    "    ),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice above that CatBoost printed scores for our `eval_set` every 50 iterations just like we said in `model_extra_params[\"fit\"]`; although, it made our results rather difficult to read, so we'll switch back to `verbose=False` during optimization.\n",
    "\n",
    "# 2. Hyperparameter Optimization\n",
    "\n",
    "Notice below that `optimizer` still recognizes the results of `experiment` as valid learning material even though their `verbose` values differ. This is because it knows that `verbose` has no effect on actual results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validated Environment with key: \"Sxkz36nLbTyi4QJM7w6wCWkbIucXm0RbzMsirLPuYmw=\"\n",
      "\u001b[31mSaved Result Files\u001b[0m\n",
      "\u001b[31m_________________________________________________________________________________________\u001b[0m\n",
      " Step |       ID |   Time |      Value |   bootstrap_type |     depth |   learning_rate | \n",
      "Experiments matching cross-experiment key/algorithm: 1\n",
      "Experiments fitting in the given space: 1\n",
      "Experiments matching current guidelines: 1\n",
      "    0 | df2982f6 | 00m00s | \u001b[35m   0.75971\u001b[0m | \u001b[32m        Bayesian\u001b[0m | \u001b[32m        5\u001b[0m | \u001b[32m         0.0500\u001b[0m | \n",
      "\u001b[31mHyperparameter Optimization\u001b[0m\n",
      "\u001b[31m_________________________________________________________________________________________\u001b[0m\n",
      " Step |       ID |   Time |      Value |   bootstrap_type |     depth |   learning_rate | \n",
      "    1 | e21de31a | 00m19s | \u001b[35m   0.86756\u001b[0m | \u001b[32m        Bayesian\u001b[0m | \u001b[32m        7\u001b[0m | \u001b[32m         0.1719\u001b[0m | \n",
      "    2 | ea45250b | 00m02s | \u001b[35m   0.89994\u001b[0m | \u001b[32m        Bayesian\u001b[0m | \u001b[32m        4\u001b[0m | \u001b[32m         0.1510\u001b[0m | \n",
      "    3 | dc911fbd | 00m19s |    0.86487 |        Bernoulli |         7 |          0.1858 | \n",
      "    4 | a344b5f7 | 00m04s |    0.89580 |         Bayesian |         5 |          0.1656 | \n",
      "    5 | 186a787e | 00m04s |    0.89318 |        Bernoulli |         5 |          0.1329 | \n",
      "    6 | 30d43916 | 00m02s | \u001b[35m   0.90540\u001b[0m | \u001b[32m       Bernoulli\u001b[0m | \u001b[32m        4\u001b[0m | \u001b[32m         0.1422\u001b[0m | \n",
      "    7 | bfe06f05 | 00m08s |    0.40699 |         Bayesian |         6 |          0.0109 | \n",
      "    8 | 514bf91a | 00m19s |    0.87122 |        Bernoulli |         7 |          0.1991 | \n",
      "    9 | 2109f8fb | 00m08s |    0.84663 |         Bayesian |         6 |          0.0910 | \n",
      "   10 | ebd045eb | 00m02s | \u001b[35m   0.92369\u001b[0m | \u001b[32m       Bernoulli\u001b[0m | \u001b[32m        4\u001b[0m | \u001b[32m         0.1999\u001b[0m | \n",
      "Optimization loop completed in 0:01:34.529025\n",
      "Best score was 0.9236872318041063 from Experiment \"ebd045eb-1224-49f7-89f3-9ff37bd0fdb7\"\n"
     ]
    }
   ],
   "source": [
    "from hyperparameter_hunter import DummyOptPro, Real, Integer, Categorical\n",
    "\n",
    "optimizer = DummyOptPro(iterations=10, random_state=777)\n",
    "\n",
    "optimizer.forge_experiment(\n",
    "    model_initializer=CatBoostRegressor,\n",
    "    model_init_params=dict(\n",
    "        iterations=100,\n",
    "        learning_rate=Real(0.001, 0.2),\n",
    "        depth=Integer(3, 7),\n",
    "        bootstrap_type=Categorical([\"Bayesian\", \"Bernoulli\"]),\n",
    "        save_snapshot=False,\n",
    "        allow_writing_files=False,\n",
    "    ),\n",
    "    model_extra_params=dict(\n",
    "        fit=dict(\n",
    "            verbose=False,\n",
    "            eval_set=[(env.validation_input, env.validation_target)],\n",
    "        ),\n",
    "    ),\n",
    ")\n",
    "\n",
    "optimizer.go()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
